package com.chance.cc.crawler.development.scripts.taobao;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.record.CrawlerResultRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.CategoryTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;

import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_Keword;

/**
 * @author bx
 * @date 2020/11/17 0017 17:18
 */
public class TaobaoSearchHtmlCrawlerScript extends CrawlerCommonScript {

    private static final Logger logger = LoggerFactory.getLogger(TaobaoSearchHtmlCrawlerScript.class);

    public static final String domain_category = "search";

    private static final String searchUrl = "https://s.taobao.com/search?q=%s&js=1" +
            "&stats_click=search_radio_all%%3A1&ie=utf8&p4ppushleft=%%2C44&sort=default&s=%d";

    private static final String searchFirstUrl = "https://s.taobao.com/search?q=%s&js=1" +
            "&stats_click=search_radio_all%%3A1&ie=utf8&p4ppushleft=%%2C44&sort=default&s=0";

    private List<String> cookies = new ArrayList<>();
    private List<String> userAgents = new ArrayList<>();

    private AtomicInteger userAgentIndex = new AtomicInteger(0);
    private Object lock = new Object();

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {

        List<CrawlerRequestRecord> crawlerRequestRecords = new ArrayList<>();

        if (!page.isDownloadSuccess()){
            logger.error("taobao search keyword proxy has error ,will retry");
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
            return crawlerRequestRecords;
        }

        String requestUrl = page.getRequest().getUrl();
        List<NameValuePair> parse = URLEncodedUtils.parse(requestUrl.split("\\?")[1], Charset.forName("utf-8"));
        int s = 0;
        String keyWord = "";
        for (NameValuePair nameValuePair : parse) {
            switch (nameValuePair.getName()){
                case "s": s = Integer.valueOf(nameValuePair.getValue()); break;
                case "q": keyWord = nameValuePair.getValue(); break;
            }
        }
        Json dataJson;
        try {
            String dataScript = extractDataInfo(page);
            if (StringUtils.isBlank(dataScript)){
                logger.error("current cookie {}",page.getRequest().getHeaders().get("cookie"));
                //可能为cookie失效导致，在此处替换cookie，流入下次循环
                String cookie = crawlerRecord.getHttpRequest().getHeaders().get("cookie");
                int nextCookieIndex = cookies.indexOf(cookie) + 1;
                if (nextCookieIndex >= cookies.size()){
                    crawlerRecord.getHttpRequest().getHeaders().put("cookie",cookies.get(0));
                } else {
                    crawlerRecord.getHttpRequest().getHeaders().put("cookie",cookies.get(nextCookieIndex));
                }
                throw new Exception("extract script information to report an error！");
            }
            dataJson = new Json(dataScript);
            dataJson.jsonPath($_type + ".mods.itemlist.data.auctions");
        } catch (Exception e) {
            logger.error("taobao search keyword download failed,error message {},page rawText detail {}",
                    e.getMessage(),page.getRawText());

            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
            return crawlerRequestRecords;
        }

        if (StringUtils.isBlank(keyWord)){
            logger.error("taobao search keyword is null,request url:" + requestUrl);
            crawlerRecord.setNeedWashPage(false);
            return null;
        }

        String nextPageurl = null;
        try {
            keyWord = URLEncoder.encode(keyWord,"utf-8");
            int totalPage = Integer.valueOf(dataJson.jsonPath($_type + ".mods.sortbar.data.pager.totalPage").get());
            int currentPage = Integer.valueOf(dataJson.jsonPath($_type + ".mods.sortbar.data.pager.currentPage").get());
            if (currentPage < totalPage){ // 100页
                nextPageurl = String.format(searchUrl,keyWord,(s + 44));
            } else {
                logger.info("taobao search html keyword {},turn page finish,total page {}", URLDecoder.decode(keyWord,"utf-8"),totalPage);
            }
            if (StringUtils.isNotBlank(nextPageurl)){
                CrawlerRequestRecord nextCrawlerRequestRecord = CrawlerRequestRecord.builder()
                        .turnPageRequest(crawlerRecord)
                        .httpUrl(nextPageurl)
                        .httpHeads(page.getRequest().getHeaders())
                        .httpHead("user-agent",getOneUserAgent())
                        .releaseTime(System.currentTimeMillis())
                        .needWashed(true)
                        .httpHeads(page.getRequest().getHeaders())
                        .build();

                crawlerRequestRecords.add(nextCrawlerRequestRecord);
            }
        } catch (UnsupportedEncodingException e) {
            logger.error(e.getMessage(),e);
        }

        try {
            Thread.sleep(RandomUtils.nextInt(3,6)*1000L);
        } catch (InterruptedException e) {
            logger.error(e.getMessage(),e);
        }

        return crawlerRequestRecords;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {

        List<CrawlerData> crawlerDataList = new ArrayList<>();

        Json dataJson = new Json(extractDataInfo(page));
        String keyword = dataJson.jsonPath($_type + ".mods.p4p.data.p4pconfig.keyword").get();

        try {
            keyword = URLDecoder.decode(keyword,"utf-8");
        } catch (UnsupportedEncodingException e) {
            logger.error(e.getMessage(),e);
        }

        List<String> itemList = dataJson.jsonPath($_type + ".mods.itemlist.data.auctions").all();
        for (String item : itemList) {
            Json itemJson = new Json(item);
            String detailUrl = itemJson.jsonPath($_type + ".detail_url").get();
            if (detailUrl.contains("item.taobao.com")){
                String itemId = itemJson.jsonPath($_type + ".nid").get();
                String sellerId = itemJson.jsonPath($_type + ".user_id").get();
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord,page)
                        .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),keyword,CrawlerEnum.CrawlerDataType.article,itemId,sellerId))
                        .releaseTime(System.currentTimeMillis())
                        .addContentKV("dataId", StringUtils.joinWith("-",crawlerRecord.getDomain(),itemId,sellerId))
                        .addContentKV("itemId", itemId)
                        .addContentKV("userId", sellerId)
                        .addContentKV("detailUrl", "https:"+detailUrl)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .build();
                crawlerData.tagsCreator().bizTags().addCustomKV(Tag_Field_Keword,keyword);
                crawlerDataList.add(crawlerData);
            }

        }

        return crawlerDataList;
    }

    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {

        List<CrawlerRecord> allItemRecords = new ArrayList<>();

        if (supportSourceRecords == null || supportSourceRecords.size() <1){
            return super.prepareRequest(requestRecord,supportSourceRecords);
        }

        CrawlerRequestRecord keywordSupportSourceRecord = null;
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            if (supportSourceRecord.getHttpRequest().getUrl().contains("/crawler/oauth/api/v1/taobao/userOauthInfos")){
                initCookieMap(supportSourceRecord); //初始cookie
            } else if (supportSourceRecord.getHttpRequest().getUrl().contains("fake-useragent.herokuapp.com/browsers")){
                initUserAgents(supportSourceRecord); //初始user agents
            } else if (supportSourceRecord.getHttpRequest().getUrl().contains("/v1/meta/taobao/keys")){
                keywordSupportSourceRecord = supportSourceRecord;
            }
        }

        if (keywordSupportSourceRecord != null){
            initAllCrawlerRecordByKeyword(requestRecord,keywordSupportSourceRecord,allItemRecords);// cookies userAgents 初始完毕后，才能初始record
        } else {
            logger.error("taobao search keywords cannot be empty!");
        }

        return allItemRecords;
    }

    private void initCookieMap(CrawlerRequestRecord supportSourceRecord){
        Set<String> cookieSets = new HashSet<>();
        try {
            HttpPage httpPage = supportSourceRecord.getInternalDownloadPage();
            Json rawText = new Json(httpPage.getRawText());
            String status = rawText.jsonPath($_type + ".status").get();
            List<String> contents = rawText.jsonPath($_type + ".content").all();

            if ("0".equals(status) && contents != null && contents.size() > 0){

                cookies.clear(); //初始新的cookies

                List<Map> userOauthInfos = new Json(contents.get(0)).toList(Map.class);
                for (Map userOauthInfo : userOauthInfos) {
                    String oauthInfo = String.valueOf(userOauthInfo.get("oauthInfo"));
                    Json oauthInfoJson = new Json(oauthInfo);
                    String category =  oauthInfoJson.jsonPath($_type + ".category").get();
                    String cookie =  oauthInfoJson.jsonPath($_type + ".cookie").get();
                    if (domain_category.equals(category)){
                        cookieSets.add(cookie);
                    }
                }
            }

        } catch (Exception e) {
            logger.error(e.getMessage(),e);
        }
        cookies.addAll(cookieSets);
    }

    private void initAllCrawlerRecordByKeyword(CrawlerRequestRecord requestRecord,
                                                              CrawlerRequestRecord supportSourceRecord,
                                                              List<CrawlerRecord> allItemRecords){
        try {
            HttpPage httpPage = supportSourceRecord.getInternalDownloadPage();
            Json rawText = new Json(httpPage.getRawText());
            String status = rawText.jsonPath($_type + ".status").get();
            List<String> contents = rawText.jsonPath($_type + ".content").all();
            if ("0".equals(status) && contents != null && contents.size() > 0){
                for (String content : contents) {
                    Json contentJson = new Json(content);
                    String keyword = contentJson.jsonPath($_type + ".keyword").get();
                    keyword = URLEncoder.encode(keyword, "utf-8");
                    CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                            .turnPageRequest(requestRecord)
                            .httpUrl(String.format(searchFirstUrl,keyword))
                            .httpHead("referer","https://s.taobao.com/")
                            .httpHead("user-agent",getOneUserAgent())
                            .httpHead("cookie",cookies.get(0)) //初始cookie 为cookie列表中的第一个
                            .releaseTime(System.currentTimeMillis())
                            .needWashed(true)
                            .build();
                    allItemRecords.add(crawlerRequestRecord);
                }
            }

        } catch (Exception e) {
            logger.error(e.getMessage(),e);
        }
    }

    private void initUserAgents( CrawlerRequestRecord supportSourceRecord){

        try {
            HttpPage httpPage = supportSourceRecord.getInternalDownloadPage();
            Json rawText = new Json(httpPage.getRawText());
            String browsers = rawText.jsonPath($_type + ".browsers").get();
            Map<String,Object> map = new Json(browsers).toObject(Map.class);
            for (Map.Entry<String, Object> entry : map.entrySet()) {
                List<String> agents = (List<String>) entry.getValue();
                userAgents.addAll(agents);
            }
        } catch (Exception e) {
            logger.error(e.getMessage(),e);
        }
    }

    private String getOneUserAgent(){

        synchronized (lock){
            if (userAgentIndex.get() >= userAgents.size()){
                userAgentIndex.set(0);
            }

            String userAgent = null;
            try {
                userAgent = userAgents.get(userAgentIndex.getAndIncrement());
            } catch (Exception e) {
                e.printStackTrace();
            }
            return userAgent;
        }
    }

    @Override
    public String domain() {
        return "taobao";
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular("https://s.taobao.com/search\\S*");
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return true;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    private void addCrawlerRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord){

        Map<String, CategoryTag> tags = crawlerRecord.getTags();
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 1;
        if (crawlerBusinessTags.hasKVTag("taobao_download_retry_count")){
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag("taobao_download_retry_count").getVal();
            if (count >= 5){
                logger.error("taobao search keyword download he number of retries exceeds the limit" +
                        ",request url {},download detail {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }
        count++;
        crawlerBusinessTags.addCustomKV("taobao_download_retry_count",count);

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpUrl(crawlerRecord.getHttpRequest().getUrl())
                .httpHeads(crawlerRecord.getHttpRequest().getHeaders())
                .httpHead("user-agent",getOneUserAgent())
                .needWashed(true)
                .releaseTime(System.currentTimeMillis())
                .build();
        crawlerRequestRecord.setTags(tags);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

    private String extractDataInfo(HttpPage page){
        List<String> scripts = page.getHtml().xpath("//script").all();
        String dataScript = null;
        for (String script : scripts) {
            script = script.trim();
            String scriptPrefix = "g_page_config = ";
            if (script.startsWith(scriptPrefix)){
                dataScript = script.substring(scriptPrefix.length(),script.lastIndexOf(";")-1);
                break;
            }
        }

        return dataScript;
    }

    public static void main(String[] args) {
        String record = "{\"collectionTime\":0,\"filter\":\"key\",\"filterInfos\":[{\"addKeyToQueue\":true,\"countTotalNum\":0,\"curCount\":1,\"date\":0,\"filter\":\"key\",\"hourFromNow\":0,\"key\":\"taobao-耐克 REACT VISION-article-624977375446-2001732744\",\"keyQueueConfig\":{\"queueContent\":\"filterKey\",\"queueGlobalKey\":\"redis-filterKey-set-filter-taobao-queue\",\"queueName\":\"filter-taobao-queue\",\"queueStorage\":\"redis\",\"queueStructure\":\"set\"}}],\"filterPipelineResult\":false,\"jsonStr\":\"{\\\"collectionTime\\\":1606455504687,\\\"content\\\":\\\"{\\\\\\\"itemId\\\\\\\":\\\\\\\"624977375446\\\\\\\",\\\\\\\"dataId\\\\\\\":\\\\\\\"taobao-624977375446-2001732744\\\\\\\",\\\\\\\"detailUrl\\\\\\\":\\\\\\\"https://item.taobao.com/item.htm?id=624977375446&ns=1&abbucket=6#detail\\\\\\\",\\\\\\\"userId\\\\\\\":\\\\\\\"2001732744\\\\\\\"}\\\",\\\"dataId\\\":\\\"taobao-耐克 REACT VISION-article-624977375446-2001732744\\\",\\\"filter\\\":\\\"key\\\",\\\"filterInfos\\\":[{\\\"addKeyToQueue\\\":true,\\\"countTotalNum\\\":0,\\\"curCount\\\":1,\\\"date\\\":0,\\\"filter\\\":\\\"key\\\",\\\"hourFromNow\\\":0,\\\"key\\\":\\\"taobao-耐克 REACT VISION-article-624977375446-2001732744\\\",\\\"keyQueueConfig\\\":{\\\"queueContent\\\":\\\"filterKey\\\",\\\"queueGlobalKey\\\":\\\"redis-filterKey-set-filter-taobao-queue\\\",\\\"queueName\\\":\\\"filter-taobao-queue\\\",\\\"queueStorage\\\":\\\"redis\\\",\\\"queueStructure\\\":\\\"set\\\"}}],\\\"releaseTime\\\":1606455512485,\\\"scheduleTime\\\":0,\\\"tags\\\":{\\\"crawler_request_tag\\\":{\\\"category\\\":\\\"crawler_request_tag\\\",\\\"labelTags\\\":{\\\"filter_request\\\":{\\\"label\\\":\\\"filter_request\\\"},\\\"result_request\\\":{\\\"label\\\":\\\"result_request\\\"}}}},\\\"url\\\":\\\"https://s.taobao.com/search?q=%E8%80%90%E5%85%8B+REACT+VISION&js=1&stats_click=search_radio_all%3A1&ie=utf8&p4ppushleft=%2C44&sort=default&s=0\\\"}\",\"recordKey\":\"taobao-耐克 REACT VISION-article-624977375446-2001732744\",\"releaseTime\":1606455512485,\"scheduleTime\":0,\"skipPipeline\":false,\"tags\":{\"crawler_request_tag\":{\"category\":\"crawler_request_tag\",\"labelTags\":{\"filter_request\":{\"label\":\"filter_request\"},\"result_request\":{\"label\":\"result_request\"}}}}}";


        CrawlerResultRecord crawlerResultRecord = JSON.parseObject(record, CrawlerResultRecord.class);

        System.out.println(crawlerResultRecord);
    }
}
